
import warnings
import os
import re
import pickle
import itertools
import numpy as np
import pandas as pd
import seaborn as sns
import plotly.express as px
from tqdm import tqdm 
from glob import glob
from time import time
from scipy import stats
from scipy.sparse import csr_matrix, hstack
from collections import Counter
from statsmodels.stats.proportion import proportion_confint
from matplotlib import pyplot as plt
from sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold, GridSearchCV, learning_curve
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression, LogisticRegressionCV, SGDClassifier
from sklearn.svm import LinearSVC
warnings.filterwarnings('ignore')
sns.set(font_scale=1.5, rc={'figure.figsize': (12,7)})
pd.set_option('display.max.columns', 25)


# Поменяйте на свой путь к данным
if 'google.colab' in str(get_ipython()):
    from google.colab import drive
    drive.mount('/content/drive')
    PATH_TO_DATA = '/content/drive/My Drive/ML\DS/Yandex_MIPT_coursera/Course6/Final1/capstone_user_identification/'
else:
    PATH_TO_DATA = 'capstone_user_identification/'


user31_data = pd.read_csv(os.path.join(PATH_TO_DATA, '10users/user0031.csv'))
user31_data.head()


def prepare_train_set(path_to_csv_files, session_length=10):
    files = sorted(glob(os.path.join(path_to_csv_files, '*.csv')))
    file_names = [name for name in sorted(os.listdir(path_to_csv_files)) if not name.startswith('.')]
    pattern = re.compile(r'\d+')
    user_ids = [int(re.search(pattern, n).group(0)) for n in file_names]
    # Создаём словарь-счётчик
    c = Counter()
    ids = []
    col_names = ['site%i' % i for i in range(1, session_length+1)]
    # Создаём пустой DataFrame 
    df = pd.DataFrame()
    for ind, file in tqdm(enumerate(files)):
        f = pd.read_csv(file)
        # Разбиваем файл на части размера session_length
        row = np.array_split(f.site.values, range(session_length, len(f), session_length))
        # Постепенно заполняем DataFrame
        df = pd.concat([df, pd.DataFrame(row)], ignore_index=True)
        # Создаём столбец user_id
        ids.extend([user_ids[ind]] * len(row))
        # Считаем частоты сайтов
        c.update(f.site)

    # Заменяем имена сайтов на их id в порядке убывания частот
    df = df.applymap(dict(zip(sorted(c, key=c.get, reverse=True), range(1, len(c)+1))).get) \
        .fillna(0).astype(int)
    df.columns = col_names
    # Добавляем столбец user_id
    df['user_id'] = ids

    # Создаём словарь частот
    freq_dict = {}
    for i, k in enumerate(c.most_common(), start=1):
        freq_dict[k[0]] = (i, c[k[0]])

    return df, freq_dict


# Файл для проверки
!cat '$PATH_TO_DATA/3users/user0001.csv'

cat: '$PATH_TO_DATA/3users/user0001.csv': Нет такого файла или каталога


train_data_toy, site_freq_3users = prepare_train_set(os.path.join(PATH_TO_DATA, '3users'), 
                                                     session_length=10)
train_data_toy

3it [00:00, 286.29it/s]


site_freq_3users

{'google.com': (1, 9),
 'oracle.com': (2, 8),
 'vk.com': (3, 3),
 'meduza.io': (4, 3),
 'mail.google.com': (5, 2),
 'football.kulichki.ru': (6, 2),
 'geo.mozilla.org': (7, 1),
 'accounts.google.com': (8, 1),
 'apis.google.com': (9, 1),
 'plus.google.com': (10, 1),
 'yandex.ru': (11, 1)}


train_data_10users, site_freq_10users = prepare_train_set(os.path.join(PATH_TO_DATA, '10users'), 
                                                     session_length=10)
train_data_150users, site_freq_150users = prepare_train_set(os.path.join(PATH_TO_DATA, '150users'), 
                                                     session_length=10)

print('Unique 10 sites sessions for 10 users: %i' % len(train_data_10users))
print('Unique 10 sites sessions for 150 users: %i' % len(train_data_150users))
print('Unique sites for 10 users: %i' % len(site_freq_10users))
print('Unique sites for 150 users: %i' % len(site_freq_150users))

10it [00:00, 64.64it/s]
150it [00:02, 57.04it/s]

Unique 10 sites sessions for 10 users: 14061
Unique 10 sites sessions for 150 users: 137019
Unique sites for 10 users: 4913
Unique sites for 150 users: 27797


top10_popular_for150 = sorted(site_freq_150users, key=lambda x: site_freq_150users.get(x)[1], reverse=True)[:10]
print('Top 10 popular sites among 150 users:')
print(*top10_popular_for150, sep='\n')

Top 10 popular sites among 150 users:
www.google.fr
www.google.com
www.facebook.com
apis.google.com
s.youtube.com
clients1.google.com
mail.google.com
plus.google.com
safebrowsing-cache.google.com
www.youtube.com


with open(os.path.join(PATH_TO_DATA, 
                       'site_freq_3users.pkl'), 'wb') as site_freq_3users_pkl:
    pickle.dump(site_freq_3users, site_freq_3users_pkl)
with open(os.path.join(PATH_TO_DATA, 
                       'site_freq_10users.pkl'), 'wb') as site_freq_10users_pkl:
    pickle.dump(site_freq_10users, site_freq_10users_pkl)
with open(os.path.join(PATH_TO_DATA, 
                       'site_freq_150users.pkl'), 'wb') as site_freq_150users_pkl:
    pickle.dump(site_freq_150users, site_freq_150users_pkl)


def get_sparse_matrix(X):
    '''
    Превращает матрицу сессий в разреженную
    '''
    rows, cols, vals = np.asarray(list(itertools.chain.from_iterable([zip([i]*len(row), \
               *np.unique(row, return_counts=True)) for i, row in enumerate(X)]))).T

    return csr_matrix((vals, (rows, cols)), shape=(len(X), np.max(X)+1))[:, 1:]


def prepare_sparse_train_set_window(path_to_csv_files, site_freq_path, 
                                    session_length=10, window_size=10):
    files = sorted(glob(os.path.join(path_to_csv_files, '*.csv')))
    file_names = [name for name in sorted(os.listdir(path_to_csv_files)) if not name.startswith('.')]
    pattern = re.compile(r'\d+')
    user_ids = [int(re.search(pattern, n).group(0)) for n in file_names]
    
    with open(site_freq_path, 'rb') as f:
        site_freq = pickle.load(f)
        
    rows, ids = [], []

    for ind, file in tqdm(enumerate(files)):
        f = pd.read_csv(file)
        # Разбиваем файл на части размера session_length со скользящим окном
        row = [f.site.values[i:i+session_length] for i in range(0, len(f), window_size)]
        # Создаём строки DataFrame
        rows.extend(row)
        # Создаём столбец user_id
        ids.extend([user_ids[ind]] * len(row))

    # Создаём DataFrame пользовательских сессий
    col_names = ['site%i' % i for i in range(1, session_length+1)]
    df = pd.DataFrame(rows, columns=col_names)
    # Заменяем имена сайтов на их id в порядке убывания частот
    df = df.applymap(lambda x: site_freq.get(x, [None])[0]) \
        .fillna(0).astype(int)
    df.columns = col_names
    X = df.values
    # Получаем разреженную матрицу
    X_sparse = get_sparse_matrix(X)
    return X_sparse, ids


%%time
for num_users in [10, 150]:
    for window_size, session_length in itertools.product([10, 7, 5], [15, 10, 7, 5]):
        if window_size <= session_length:
            X_sparse, y = prepare_sparse_train_set_window(
                          os.path.join(PATH_TO_DATA,'{0}users'.format(num_users)), 
                          os.path.join(PATH_TO_DATA,'site_freq_{0}users.pkl'.format(num_users)),
                          session_length=session_length, window_size=window_size)

            with open(os.path.join(PATH_TO_DATA, \
                      'X_sparse_{0}users_s{1}_w{2}.pkl' \
                      .format(num_users, session_length, window_size)), 'wb') as fx:
                pickle.dump(X_sparse, fx)

            with open(os.path.join(PATH_TO_DATA, \
                      'y_{0}users_s{1}_w{2}.pkl' \
                      .format(num_users, session_length, window_size)), 'wb') as fy:
                pickle.dump(y, fy)

10it [00:00, 68.66it/s]
10it [00:00, 73.10it/s]
10it [00:00, 60.76it/s]
10it [00:00, 59.32it/s]
10it [00:00, 59.65it/s]
10it [00:00, 49.81it/s]
10it [00:00, 49.87it/s]
10it [00:00, 49.01it/s]
10it [00:00, 48.32it/s]
150it [00:01, 102.10it/s]
150it [00:01, 104.56it/s]
150it [00:01, 90.35it/s]
150it [00:01, 87.96it/s]
150it [00:01, 82.05it/s]
150it [00:02, 68.93it/s]
150it [00:02, 72.06it/s]
150it [00:02, 72.81it/s]
150it [00:02, 69.76it/s]

CPU times: user 1min 30s, sys: 1.89 s, total: 1min 32s
Wall time: 1min 33s


# Общий вид данных
train_data_10users.head()


train_data_10users.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 14061 entries, 0 to 14060
Data columns (total 11 columns):
 #   Column   Non-Null Count  Dtype
---  ------   --------------  -----
 0   site1    14061 non-null  int64
 1   site2    14061 non-null  int64
 2   site3    14061 non-null  int64
 3   site4    14061 non-null  int64
 4   site5    14061 non-null  int64
 5   site6    14061 non-null  int64
 6   site7    14061 non-null  int64
 7   site8    14061 non-null  int64
 8   site9    14061 non-null  int64
 9   site10   14061 non-null  int64
 10  user_id  14061 non-null  int64
dtypes: int64(11)
memory usage: 1.2 MB


train_data_10users['user_id'].value_counts()

128    2796
39     2204
207    1868
127    1712
237    1643
33     1022
50      802
31      760
100     720
241     534
Name: user_id, dtype: int64


num_unique_sites = [np.unique(train_data_10users.values[i, :-1]).shape[0] 
                    for i in range(train_data_10users.shape[0])]
pd.Series(num_unique_sites).value_counts()

7     2308
6     2197
8     2046
5     1735
9     1394
2     1246
4     1163
3      894
10     651
1      427
dtype: int64


stats.probplot(num_unique_sites, plot=plt, rvalue=True);
_, p = stats.shapiro(num_unique_sites)
print(f'p-value for Shapiro-Wilk test = {p}')

p-value for Shapiro-Wilk test = 0.0


has_two_similar = (np.array(num_unique_sites) < 10).astype('int')
p_val = stats.binom_test(np.sum(has_two_similar), len(has_two_similar), p=0.95, alternative='greater')
print(f'Binomial test p-value = {p_val:.5f}')

wilson_interval = proportion_confint(np.sum(has_two_similar), len(has_two_similar), method='wilson')
print('Wilson confidence interval = [{0:.3f}, {1:.3f}]'.format(*wilson_interval))

Binomial test p-value = 0.02208
Wilson confidence interval = [0.950, 0.957]


def get_bootstrap_samples(data, n_samples, random_seed=17):
    np.random.seed(random_seed)
    indices = np.random.randint(0, len(data), (n_samples, len(data)))
    samples = data[indices]
    return samples


def stat_intervals(stat, alpha):
    boundaries = np.percentile(stat, 
                 [100 * alpha / 2., 100 * (1 - alpha / 2.)])
    return boundaries


site_freq_10users_np = np.array(list(site_freq_10users.values()))[:, 1]
bsamples = get_bootstrap_samples(site_freq_10users_np, len(site_freq_10users_np)).mean(axis=1)
conf_int = stat_intervals(bsamples, 0.05)
print('Bootstrap confidence interval for site mean frequency = [{0:.3f}, {1:.3f}]'.format(*conf_int))

Bootstrap confidence interval for site mean frequency = [22.515, 35.763]


def prepare_train_set_with_fe(path_to_csv_files, site_freq_path, feature_names,
                                    session_length=10, window_size=10):
    
    files = sorted(glob(os.path.join(path_to_csv_files, '*.csv')))
    file_names = [name for name in sorted(os.listdir(path_to_csv_files)) if not name.startswith('.')]
    pattern = re.compile(r'\d+')
    user_ids = [int(re.search(pattern, n).group(0)) for n in file_names]
    
    with open(site_freq_path, 'rb') as f:
        site_freq = pickle.load(f)

    sessions = []
    durations = []
    timespans = []
    unique_sites = []
    start_hours = []
    days_of_week = []
    ids = []

    for ind, file in tqdm(enumerate(files)):
        f = pd.read_csv(file, parse_dates=[0])
        # Разбиваем файл на части размера "session_length" скользящим окном
        row = [f.site.values[i:i+session_length] for i in range(0, len(f), window_size)]
        # Получаем количество уникальных сайтов в каждой сессии в файле
        num_unique = list(map(lambda x: len(set(x)), row))
        # Получаем первое и последнее время для каждой сессии
        timestamps = [f.timestamp.values[i:i+session_length]
                       for i in range(0, len(f), window_size)]
        # Получаем продолжительности сессий, час начала и день недели
        diffs = list(map(lambda x: np.diff(x) // np.timedelta64(1, 's'), timestamps))
        timespan = [(x[-1] - x[0]) // np.timedelta64(1, 's') for x in timestamps]
        hour = list(map(lambda x: pd.Timestamp(x[0]).hour, timestamps))
        dow = list(map(lambda x: pd.Timestamp(x[0]).dayofweek, timestamps))
        
        # Создаём строки и столбцы будущего DataFrame
        sessions.extend(row)
        durations.extend(diffs)
        timespans.extend(timespan)
        unique_sites.extend(num_unique)
        start_hours.extend(hour)
        days_of_week.extend(dow)
        ids.extend([user_ids[ind]] * len(row))

    # Создаём DataFrame из сессий и заменяем имена сайтов их id в порядке убывания частот
    df = pd.DataFrame(sessions)
    df = df.applymap(lambda x: site_freq.get(x, [None])[0]).fillna(0).astype(int)

    # Добавляем остальные созданные столбцы и даём им имена
    durations_df = pd.DataFrame(durations).fillna(0).astype(int)
    other_features_df = pd.DataFrame(np.stack([timespans, unique_sites,
                                                start_hours, days_of_week, ids], axis=1)) 

    df = pd.concat([df, durations_df, other_features_df], axis=1)
    df.columns = feature_names

    return df


feature_names = ['site' + str(i) for i in range(1,11)] + \
                ['time_diff' + str(j) for j in range(1,10)] + \
                ['session_timespan', '#unique_sites', 'start_hour', 
                 'day_of_week', 'target']


%%time
train_data_10users = prepare_train_set_with_fe(
                         path_to_csv_files=os.path.join(PATH_TO_DATA, '10users'), 
                         site_freq_path=os.path.join(PATH_TO_DATA, 'site_freq_10users.pkl'),
                         feature_names=feature_names,
                         session_length=10,
                         window_size=10)

10it [00:00, 20.86it/s]

CPU times: user 599 ms, sys: 187 µs, total: 599 ms
Wall time: 597 ms


train_data_10users.head()


%%time
train_data_150users = prepare_train_set_with_fe(
                         path_to_csv_files=os.path.join(PATH_TO_DATA, '150users'), 
                         site_freq_path=os.path.join(PATH_TO_DATA, 'site_freq_150users.pkl'),
                         feature_names=feature_names,
                         session_length=10,
                         window_size=10)

150it [00:04, 30.29it/s]

CPU times: user 6.01 s, sys: 31.1 ms, total: 6.04 s
Wall time: 6.11 s


train_data_150users.head()


id_name_dict = {128: 'Mary-Kate', 39: 'Ashley', 207: 'Lindsey', 127: 'Naomi', 237: 'Avril',
               33: 'Bob', 50: 'Bill', 31: 'John', 100: 'Dick', 241: 'Ed'}

color_dic = {'Mary-Kate': 'pink', 'Ashley': 'darkviolet', 'Lindsey':'blueviolet', 
             'Naomi': 'hotpink', 'Avril': 'orchid', 
             'Bob': 'firebrick', 'Bill': 'gold', 'John': 'forestgreen', 
             'Dick': 'slategrey', 'Ed':'brown'}

train_data_10users['target'] = train_data_10users['target'].map(id_name_dict)


def draw_hist(data, color_list, hue=None, xlabel='x', ylabel='y', title=None, legend=False):
    '''
    Рисует ploly гистограмму
    '''
    fig = px.histogram(data, color=hue, color_discrete_sequence=color_list, width=800, height=500)
    # Рисуем белый контур у бинов, чтобы выглядело красивее
    fig.update_traces(marker=dict(line=dict(width=1, color='white')))
    
    fig.update_layout(showlegend=legend,
                      xaxis=dict(title=dict(text=xlabel)),
                      yaxis=dict(title=dict(text=ylabel)),
                      title=dict(text=title, x=0.5))

    return fig


def draw_10_plots(data, names, colors, xlabel=None, ylabel=None, custom_ticks=None, title=None):
    sns.set()
    fig = plt.figure(figsize=(22, 10))
    for idx, name in enumerate(names, start=1):
        ax = fig.add_subplot(3, 4, idx)
        sns.histplot(data[name], kde=False, color=colors[name], label=name, discrete=True)
        plt.xlabel(xlabel)
        plt.ylabel(ylabel)
        plt.legend()
        
        if custom_ticks:
            plt.xticks(list(custom_ticks.keys()), list(custom_ticks.values()))
            
        if title:
            plt.title(title + ' для ' + name)
            
    plt.subplots_adjust(wspace=0.5, hspace=0.5)


limited_timespan = train_data_10users['session_timespan'][train_data_10users['session_timespan'] <= 200]
draw_hist(data=limited_timespan,
          color_list=['darkviolet'],
          xlabel='Длина сессии, с.',
          ylabel='Количество сессий',
          title='Гистограмма распределения длины сессий')


draw_hist(data=train_data_10users['#unique_sites'],
          color_list=['aqua'],
          xlabel='Количество уникальных сайтов',
          ylabel='Количество сессий',
          title='Гистограмма распределения числа уникальных сайтов в сессии')


user_unique_sites = train_data_10users.groupby('target')['#unique_sites'].agg(list)
user_names = user_unique_sites.index
draw_10_plots(user_unique_sites, user_names, color_dic,
              xlabel='Количество уникальных сайтов',
              ylabel='Количество сессий')


draw_hist(train_data_10users['start_hour'], color_list=['darkgreen'],
         xlabel='Час начала сессии', ylabel='Количество сессий',
         title='Гистограмма распределения часа начала сессии')


user_start_hour = train_data_10users.groupby('target')['start_hour'].agg(list)
draw_10_plots(user_start_hour, user_names, color_dic,
              xlabel='Час начала сессии',
              ylabel='Количество сессий',
              custom_ticks={i: i for i in range(8, 23, 2)})


fig = draw_hist(train_data_10users['day_of_week'], color_list=['sienna'],
          xlabel='День недели', ylabel='Количество сессий',
          title='Гистограмма распределения дня недели')

fig.update_layout(
    xaxis = dict(
        tickmode = 'array',
        tickvals = [0, 1, 2, 3, 4, 5, 6],
        ticktext = ['Пн', 'Вт', 'Ср', 'Чт', 'Пт', 'Сб', 'Вс']
    )
)


ticks = {0:'Пн', 1:'Вт', 2:'Ср', 3:'Чт', 4:'Пт', 5:'Сб', 6:'Вс'}
user_dow = train_data_10users.groupby('target')['day_of_week'].agg(list)
draw_10_plots(user_dow, user_names, color_dic,
              xlabel='День недели',
              ylabel='Количество сессий',
              title='Распределение дня недели',
              custom_ticks=ticks)


fig = px.histogram(train_data_10users, x='target', y='day_of_week', color='target',
                   color_discrete_map=color_dic, histfunc='count',
                   category_orders={'target': sorted(user_names)},
                   width=800, height=500)

fig.update_layout(showlegend=False,
                  xaxis=dict(title=dict(text='Пользователь')),
                  yaxis=dict(title=dict(text='Количество сессий')),
                  title=dict(text='Гистограмма распределения количества сессий по пользователю', x=0.5))


top10_sites, top10_freqs = [], []
for i, (site, freq) in enumerate(site_freq_10users.items()):
    if i == 10:
        break
    top10_sites.append(site)
    top10_freqs.append(freq[1])


fig = px.bar(x=top10_sites, y=top10_freqs, color=top10_sites,
             width=1000, height=600)

fig.update_layout(showlegend=True,
                  xaxis=dict(title=dict(text='Сайт')),
                  yaxis=dict(title=dict(text='Частота')),
                  title=dict(text='Частоты посещений топ-10 сайтов', x=0.25))

fig.show()


train_data_10users.start_hour.describe(percentiles=[])

count    14061.000000
mean        13.768366
std          3.681279
min          7.000000
50%         14.000000
max         23.000000
Name: start_hour, dtype: float64


train_data_150users.start_hour.describe(percentiles=[])

count    137019.000000
mean         12.839373
std           3.288743
min           7.000000
50%          13.000000
max          23.000000
Name: start_hour, dtype: float64


def build_time_of_day(data):
    '''
    Takes start hour and converts it to time_of_day
    0-6: night
    7-11: morning
    12-17: afternoon
    18-23: evening
    Returns new Series time_of_day
    '''
    tod = pd.Series([None] * len(data), name='time_of_day')
    tod[(data <= 23) & (data >= 18)] = 'evening'
    tod[(data <= 17) & (data >= 12)] = 'afternoon'
    tod[(data <= 11) & (data >= 7)] = 'morning'
    tod[(data <= 6) & (data >= 0)] = 'night'
    return tod


# Применим функцию для данных 10 пользователей и посмотрим картинки
time_of_day10 = build_time_of_day(train_data_10users.start_hour)
draw_hist(time_of_day10, hue=time_of_day10, color_list=['darkcyan', 'gold', 'orange'],
          xlabel='Время суток', ylabel='Количество сессий',
          title='Зависимость количества сессий от времени суток')


tod_df = pd.concat([time_of_day10, train_data_10users.target], axis=1)
user_tod = tod_df.groupby('target')['time_of_day'].agg(list)
draw_10_plots(user_tod.map(sorted), names=user_names, colors=color_dic,
              xlabel='Время суток', ylabel='Количество сессий',
              title='Распределение времени суток')


def build_is_weekend(data):
    '''
    Takes day of week and checks if it's weekend day
    0-4: not weekend (0)
    5-6: weekend (1)
    Returns new Series is_weekend
    '''
    isw = pd.Series([0] * len(data), name='is_weekend', dtype=int)
    isw[(data <= 6) & (data >= 5)] = 1
    return isw


is_weekend10 = build_is_weekend(train_data_10users.day_of_week)
isw_df = pd.concat([is_weekend10, train_data_10users.target], axis=1)
user_isw = isw_df.groupby('target')['is_weekend'].agg(list)
draw_10_plots(user_isw, names=user_names, colors=color_dic,
              xlabel='Выходной', ylabel='Количество сессий',
              title='Распределение сессий по выходным',
              custom_ticks={0: 'Нет', 1: 'Да'})


fig = draw_hist(is_weekend10, hue=is_weekend10, color_list=['cornflowerblue', 'salmon'],
                xlabel='Выходной', ylabel='Количество сессий',
                title='Распределение сессий по выходным')

fig.update_layout(
    xaxis = dict(
        tickmode = 'array',
        tickvals = [0, 1],
        ticktext = ['Нет', 'Да']
    )
)


def build_num_of_social(data, site_freq):
    '''
    Takes info about all visited sites in all user sessions and dict of sites frequencies.
    Returns Series with total number of social network sites visited (not unique!)
    Social networks: facebook, youtube, instagram and twitter
    '''
    # Найдём id сайтов соц. сетей
    pattern = re.compile(r'facebook|youtube|instagram|twitter')
    sites_ids = [val[0] for key, val in site_freq.items() if re.findall(pattern, key)]

    counts = data.apply(lambda x: dict(zip(*np.unique(x, return_counts=True))),axis=1)
    num_of_social = counts.apply(lambda x: np.sum([x.get(y) for y in sites_ids if y in x], dtype=int))
    num_of_social.name = '#social_networks'
    return num_of_social


num_of_social10 = build_num_of_social(train_data_10users.loc[:, 'site1':'site10'],
                                      site_freq_10users)
nos_df = pd.concat([num_of_social10, train_data_10users.target], axis=1)


draw_hist(num_of_social10, color_list=['deepSkyBlue'],
          xlabel='Количество сайтов соц. сетей в сессии',
          ylabel='Количество сессий',
          title='Распределение количества сайтов соц.сетей в сессии')


user_nos = nos_df.groupby('target')['#social_networks'].agg(list)
draw_10_plots(user_nos, names=user_names, colors=color_dic,
              xlabel='Количество сайтов соц. сетей в сессии', ylabel='Количество сессий',
              title='Распределение соц.сетей в сессии',
              custom_ticks={i: i for i in range(0, 11, 2)})


def build_pop_total_ratio(sites, time_diffs, timespans, site_freq):
    '''
    Takes info about all visited sites in all user sessions, time spent on these sites,
    total session timespan and dict of sites frequencies.
    Returns Series with ratio of time spent on 30 most popular sites + social networks
    to total session timespan.
    Social networks: facebook, youtube, instagram and twitter
    '''
    from collections import defaultdict

    # Найдём id сайтов соц. сетей
    pattern = re.compile(r'facebook|youtube|instagram|twitter')
    sites_ids = [val[0] for key, val in site_freq.items() if re.findall(pattern, key)]

    # Поставим в соответствие каждому сайту время, в течении которого на нём находился пользователь
    add_zeros = np.hstack([time_diffs.values, np.zeros(time_diffs.shape[0]).reshape(-1, 1)])
    zipped = np.dstack([sites.values, add_zeros])

    # Создаём столбец данных
    s = pd.Series(data=[None]*len(zipped), name='pop_total_ratio')
    for i, row in enumerate(zipped):
        d = defaultdict(int)
        for k, v in row:
            d[k] += v
        res = np.sum([d.get(x) for x in set(range(30)).union(sites_ids) if x in d])
        s[i] = res / timespans[i]
    s.fillna(0, inplace=True)
    return s


sites_df = train_data_10users.loc[:, 'site1':'site10']
time_diffs_df = train_data_10users.loc[:, 'time_diff1':'time_diff9']
timespans = train_data_10users.session_timespan


pop_total10 = build_pop_total_ratio(sites_df, time_diffs_df, timespans, site_freq_10users)


draw_hist(pop_total10, color_list=['greenYellow'],
          xlabel='Отношение времени на популярных сайтах и соц.сетях к общему времени сессии',
          ylabel='Количество сессий',
          title='Распределение отношения времени на популярных сайтах и соц.сетях к общему времени сессии')


ptr_df = pd.concat([pop_total10, train_data_10users.target], axis=1)
user_ptr = ptr_df.groupby('target')['pop_total_ratio'].agg(list)


fig = plt.figure(figsize=(22, 10))
for idx, name in enumerate(user_names, start=1):
    ax = fig.add_subplot(3, 4, idx)
    sns.histplot(user_ptr[name], kde=False, color=color_dic[name], label=name)
    plt.xlabel('Отношение времени')
    plt.ylabel('Количество сессий')
    plt.legend()
    plt.title('Распределение отношения времени для ' + name)
        
plt.subplots_adjust(wspace=0.5, hspace=0.5)


def feature_engineering(data, site_freq_path):
    '''
    Use as an addition to function prepare_train_set_with_fe
    '''
    # Загружаем словарь частот сайтов
    with open(site_freq_path, 'rb') as f:
        site_freq = pickle.load(f)

    # Приготовим нужные части данных 
    sites = data.loc[:, 'site1':'site10']
    diffs = data.loc[:, 'time_diff1':'time_diff9']
    timespans = data.session_timespan
    start_hours = data.start_hour
    days_of_week = data.day_of_week

    # Строим признак времени суток
    time_of_day = build_time_of_day(start_hours)
    # Строим признак выходного дня
    is_weekend = build_is_weekend(days_of_week)
    # Строим признак количества посещённых соц. сетей
    num_of_social = build_num_of_social(sites, site_freq)
    # Строим признак отношения времени проведённого на популярных сайтах к времени сессии
    pop_total_ratio = build_pop_total_ratio(sites, diffs, timespans, site_freq)

    return pd.concat([time_of_day, is_weekend, num_of_social, pop_total_ratio], axis=1)


new_features_10users = feature_engineering(train_data_10users, os.path.join(PATH_TO_DATA, 'site_freq_10users.pkl'))


%%time
new_features_150users = feature_engineering(train_data_150users, os.path.join(PATH_TO_DATA, 'site_freq_150users.pkl'))

CPU times: user 13 s, sys: 193 ms, total: 13.2 s
Wall time: 13.3 s


new_features_150users.head()


with open(os.path.join(PATH_TO_DATA, 
         'X_sparse_10users.pkl'), 'rb') as X_sparse_10users_pkl:
    X_sparse_10users = pickle.load(X_sparse_10users_pkl)
with open(os.path.join(PATH_TO_DATA, 
                       'y_10users.pkl'), 'rb') as y_10users_pkl:
    y_10users = pickle.load(y_10users_pkl)


X_train, X_valid, y_train, y_valid = train_test_split(X_sparse_10users, y_10users, 
                                                      test_size=0.3, 
                                                     random_state=17, stratify=y_10users)


skf = StratifiedKFold(n_splits=3, shuffle=True, random_state=17)


def plot_validation_curves(param_values, grid_cv_results_):
    train_mu, train_std = grid_cv_results_['mean_train_score'], grid_cv_results_['std_train_score']
    valid_mu, valid_std = grid_cv_results_['mean_test_score'], grid_cv_results_['std_test_score']
    train_line = plt.plot(param_values, train_mu, '-', label='train', color='green')
    valid_line = plt.plot(param_values, valid_mu, '-', label='test', color='red')
    plt.fill_between(param_values, train_mu - train_std, train_mu + train_std, edgecolor='none',
                     facecolor=train_line[0].get_color(), alpha=0.2)
    plt.fill_between(param_values, valid_mu - valid_std, valid_mu + valid_std, edgecolor='none',
                     facecolor=valid_line[0].get_color(), alpha=0.2)
    plt.legend()


knn = KNeighborsClassifier(n_neighbors=100, n_jobs=-1)
forest = RandomForestClassifier(n_estimators=100, random_state=17, n_jobs=-1)
logit = LogisticRegression(random_state=17, n_jobs=-1)
svm = LinearSVC(random_state=17)
sgd_logit = SGDClassifier(loss='log', random_state=17, n_jobs=-1)
sgd_svm = SGDClassifier(loss='hinge', random_state=17, n_jobs=-1)


def get_model_scores(model, X_train, y_train, X_valid, y_valid, cv, name=None):
    cv_scores = cross_val_score(model, X_train, y_train, scoring='accuracy', cv=cv, n_jobs=-1)
    mean_cv_score = np.mean(cv_scores)
    time1 = time()
    model.fit(X_train, y_train)
    time2 = time()
    val_pred = model.predict(X_valid)
    val_score = accuracy_score(y_valid, val_pred)
    print('-' * 25)
    print(name)
    print('-' * 25)
    print(f'CV accuracy scores = {cv_scores}')
    print(f'Mean CV accuracy score = {mean_cv_score:.5f}')
    print(f'Validation set accuracy score = {val_score:.5f}')
    print(f'Fitting time: {(time2 - time1):.2f} s')


models_dict = {'kNN': knn, 'Random Forest': forest,
               'Logistic Regression': logit, 'SVM': svm,
               'Logistic Regression (SGD)': sgd_logit,
               'SVM (SGD)': sgd_svm}


for name, model in models_dict.items():
    get_model_scores(model, X_train, y_train, X_valid, y_valid, cv=skf, name=name)

-------------------------
kNN
-------------------------
CV accuracy scores = [0.56598598 0.55409936 0.55792683]
Mean CV accuracy score = 0.55934
Validation set accuracy score = 0.58402
Fitting time: 0.00 s
-------------------------
Random Forest
-------------------------
CV accuracy scores = [0.72112161 0.7089302  0.72256098]
Mean CV accuracy score = 0.71754
Validation set accuracy score = 0.73122
Fitting time: 2.54 s
-------------------------
Logistic Regression
-------------------------
CV accuracy scores = [0.76104846 0.74824749 0.77256098]
Mean CV accuracy score = 0.76062
Validation set accuracy score = 0.77672
Fitting time: 1.21 s
-------------------------
SVM
-------------------------
CV accuracy scores = [0.75068577 0.73270344 0.7695122 ]
Mean CV accuracy score = 0.75097
Validation set accuracy score = 0.77696
Fitting time: 1.42 s
-------------------------
Logistic Regression (SGD)
-------------------------
CV accuracy scores = [0.75586711 0.74062786 0.77530488]
Mean CV accuracy score = 0.75727
Validation set accuracy score = 0.77672
Fitting time: 0.20 s
-------------------------
SVM (SGD)
-------------------------
CV accuracy scores = [0.74946663 0.72935081 0.7597561 ]
Mean CV accuracy score = 0.74619
Validation set accuracy score = 0.76558
Fitting time: 0.21 s


Cs = np.logspace(-4, 2, 10)
logit_cv = LogisticRegressionCV(Cs=Cs, cv=skf, scoring='accuracy', random_state=17, n_jobs=-1)
svm_cv = GridSearchCV(svm, param_grid={'C': Cs}, cv=skf, scoring='accuracy', n_jobs=-1)


%%time
logit_cv.fit(X_train, y_train)

CPU times: user 5.15 s, sys: 313 ms, total: 5.47 s
Wall time: 9.88 s

LogisticRegressionCV(Cs=array([1.00000000e-04, 4.64158883e-04, 2.15443469e-03, 1.00000000e-02,
       4.64158883e-02, 2.15443469e-01, 1.00000000e+00, 4.64158883e+00,
       2.15443469e+01, 1.00000000e+02]),
                     cv=StratifiedKFold(n_splits=3, random_state=17, shuffle=True),
                     n_jobs=-1, random_state=17, scoring='accuracy')


logit_cv_best_C = logit_cv.C_[0]
logit_cv_scores = np.mean(list(logit_cv.scores_.values())[0], axis=0)


%%time
svm_cv.fit(X_train, y_train)

CPU times: user 986 ms, sys: 16.7 ms, total: 1 s
Wall time: 6.79 s

GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=17, shuffle=True),
             estimator=LinearSVC(random_state=17), n_jobs=-1,
             param_grid={'C': array([1.00000000e-04, 4.64158883e-04, 2.15443469e-03, 1.00000000e-02,
       4.64158883e-02, 2.15443469e-01, 1.00000000e+00, 4.64158883e+00,
       2.15443469e+01, 1.00000000e+02])},
             scoring='accuracy')


svm_cv_best_C = svm_cv.best_params_['C']
svm_cv_scores = svm_cv.cv_results_['mean_test_score']


fig = px.line(x=Cs, y=[logit_cv_scores, svm_cv_scores], log_x=True,
              labels={'x': 'Cs', 'value': 'Accuracy', 'variable': 'Model'},
              title='Валидационные кривые',
              width=800, height=500)

fig.update_traces(mode='lines+markers')
fig.update_layout(title=dict(x=0.5,
                             xanchor='center'),
                  legend=dict(yanchor="bottom",
                              y=0.01,
                              xanchor="right",
                              x=0.99,
                              bgcolor='rgba(0, 0, 0, 0)'))

fig.data[0].name = 'Logistic Regression'
fig.data[1].name = 'SVM'
fig


logit_cv_val_acc = accuracy_score(y_valid, logit_cv.predict(X_valid))
svm_cv_val_acc = accuracy_score(y_valid, svm_cv.predict(X_valid))
print(f'Best Log. Reg.: C = {logit_cv_best_C:.3f}; CV acc. = {logit_cv_scores.max():.5f}; Val. acc. = {logit_cv_val_acc:.5f}')
print(f'Best SVM: C = {svm_cv_best_C:.3f}; CV acc. = {svm_cv.best_score_:.5f}; Val. acc. = {svm_cv_val_acc:.5f}')

Best Log. Reg.: C = 1.000; CV acc. = 0.76082; Val. acc. = 0.77530
Best SVM: C = 0.215; CV acc. = 0.76621; Val. acc. = 0.78336


Cs = np.linspace(0.1, 5, 50)
logit_cv = LogisticRegressionCV(Cs=Cs, cv=skf, scoring='accuracy', random_state=17, n_jobs=-1)
svm_cv = GridSearchCV(svm, param_grid={'C': Cs}, cv=skf, scoring='accuracy', n_jobs=-1)


%%time
logit_cv.fit(X_train, y_train)

CPU times: user 3.98 s, sys: 243 ms, total: 4.22 s
Wall time: 1min 4s

LogisticRegressionCV(Cs=array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1. , 1.1, 1.2, 1.3,
       1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2. , 2.1, 2.2, 2.3, 2.4, 2.5, 2.6,
       2.7, 2.8, 2.9, 3. , 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9,
       4. , 4.1, 4.2, 4.3, 4.4, 4.5, 4.6, 4.7, 4.8, 4.9, 5. ]),
                     cv=StratifiedKFold(n_splits=3, random_state=17, shuffle=True),
                     n_jobs=-1, random_state=17, scoring='accuracy')


%%time
svm_cv.fit(X_train, y_train)

CPU times: user 1.09 s, sys: 39.9 ms, total: 1.13 s
Wall time: 44.7 s

GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=17, shuffle=True),
             estimator=LinearSVC(random_state=17), n_jobs=-1,
             param_grid={'C': array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1. , 1.1, 1.2, 1.3,
       1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2. , 2.1, 2.2, 2.3, 2.4, 2.5, 2.6,
       2.7, 2.8, 2.9, 3. , 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9,
       4. , 4.1, 4.2, 4.3, 4.4, 4.5, 4.6, 4.7, 4.8, 4.9, 5. ])},
             scoring='accuracy')


logit_cv_best_C = logit_cv.C_[0]
logit_cv_scores = np.mean(list(logit_cv.scores_.values())[0], axis=0)

svm_cv_best_C = svm_cv.best_params_['C']
svm_cv_scores = svm_cv.cv_results_['mean_test_score']

logit_cv_val_acc = accuracy_score(y_valid, logit_cv.predict(X_valid))
svm_cv_val_acc = accuracy_score(y_valid, svm_cv.predict(X_valid))
print(f'Best Log. Reg.: C = {logit_cv_best_C:.3f}; CV acc. = {logit_cv_scores.max():.5f}; Val. acc. = {logit_cv_val_acc:.5f}')
print(f'Best SVM: C = {svm_cv_best_C:.3f}; CV acc. = {svm_cv.best_score_:.5f}; Val. acc. = {svm_cv_val_acc:.5f}')

Best Log. Reg.: C = 1.700; CV acc. = 0.76153; Val. acc. = 0.77957
Best SVM: C = 0.100; CV acc. = 0.76692; Val. acc. = 0.77981


fig = px.line(x=Cs, y=[logit_cv_scores, svm_cv_scores],
              labels={'x': 'Cs', 'value': 'Accuracy', 'variable': 'Model'},
              title='Валидационные кривые',
              width=800, height=500)

fig.update_traces(mode='lines+markers')
fig.update_layout(title=dict(x=0.5,
                             xanchor='center'),
                  legend=dict(yanchor="top",
                              y=0.99,
                              xanchor="right",
                              x=0.99,
                              bgcolor='rgba(0, 0, 0, 0)'))

fig.data[0].name = 'Logistic Regression'
fig.data[1].name = 'SVM'
fig


def model_assessment(estimator, path_to_X_pickle, path_to_y_pickle, cv, random_state=17, test_size=0.3):
    '''
    Estimates CV-accuracy for (1 - test_size) share of (X_sparse, y) 
    loaded from path_to_X_pickle and path_to_y_pickle and holdout accuracy for (test_size) share of (X_sparse, y).
    The split is made with stratified train_test_split with params random_state and test_size.
    
    :param estimator – Scikit-learn estimator (classifier or regressor)
    :param path_to_X_pickle – path to pickled sparse X (instances and their features)
    :param path_to_y_pickle – path to pickled y (responses)
    :param cv – cross-validation as in cross_val_score (use StratifiedKFold here)
    :param random_state –  for train_test_split
    :param test_size –  for train_test_split
    
    :returns mean CV-accuracy for (X_train, y_train) and accuracy for (X_valid, y_valid) where (X_train, y_train)
    and (X_valid, y_valid) are (1 - test_size) and (testsize) shares of (X_sparse, y).
    '''
    start_time = time()
    
    with open(path_to_X_pickle, 'rb') as fx:
        X = pickle.load(fx)
        
    with open(path_to_y_pickle, 'rb') as fy:
        y = pickle.load(fy)
        
    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=test_size, random_state=random_state,
                                                        stratify=y)
    
    cv_scores = cross_val_score(estimator, X_train, y_train, cv=cv, n_jobs=-1)
    cv_accuracy = cv_scores.mean()
    
    estimator.fit(X_train, y_train)
    predictions = estimator.predict(X_valid)
    val_accuracy = accuracy_score(y_valid, predictions)
    
    return cv_accuracy, val_accuracy, '%.2f s' % (time() - start_time)


!cp $PATH_TO_DATA/X_sparse_10users.pkl $PATH_TO_DATA/X_sparse_10users_s10_w10.pkl 
!cp $PATH_TO_DATA/X_sparse_150users.pkl $PATH_TO_DATA/X_sparse_150users_s10_w10.pkl 
!cp $PATH_TO_DATA/y_10users.pkl $PATH_TO_DATA/y_10users_s10_w10.pkl 
!cp $PATH_TO_DATA/y_150users.pkl $PATH_TO_DATA/y_150users_s10_w10.pkl

cp: не удалось выполнить stat для '$PATH_TO_DATA/X_sparse_10users.pkl': Нет такого файла или каталога
cp: не удалось выполнить stat для '$PATH_TO_DATA/X_sparse_150users.pkl': Нет такого файла или каталога
cp: не удалось выполнить stat для '$PATH_TO_DATA/y_10users.pkl': Нет такого файла или каталога
cp: не удалось выполнить stat для '$PATH_TO_DATA/y_150users.pkl': Нет такого файла или каталога


estimator = svm_cv.best_estimator_

for window_size, session_length in itertools.product([10, 7, 5], [15, 10, 7, 5]):
    if window_size <= session_length:
        X_file_name = f'X_sparse_10users_s{session_length}_w{window_size}.pkl'
        y_file_name = f'y_10users_s{session_length}_w{window_size}.pkl'
        path_to_X_pkl = os.path.join(PATH_TO_DATA, X_file_name)
        path_to_y_pkl = os.path.join(PATH_TO_DATA, y_file_name)
        print(f'Session length = {session_length}')
        print(f'Window size = {window_size}')
        print('CV acc. = {0:.5f}; Val. acc. = {1:.5f}; Elapsed time = {2}' \
              .format(*model_assessment(estimator, 
                                        path_to_X_pkl, path_to_y_pkl, skf,
                                        random_state=17, test_size=0.3)))
        print('_' * 20)

Session length = 15
Window size = 10
CV acc. = 0.82412; Val. acc. = 0.84048; Elapsed time = 1.56 s
____________________
Session length = 10
Window size = 10
CV acc. = 0.76692; Val. acc. = 0.77981; Elapsed time = 0.78 s
____________________
Session length = 15
Window size = 7
CV acc. = 0.84964; Val. acc. = 0.85382; Elapsed time = 2.00 s
____________________
Session length = 10
Window size = 7
CV acc. = 0.79829; Val. acc. = 0.80720; Elapsed time = 1.11 s
____________________
Session length = 7
Window size = 7
CV acc. = 0.75448; Val. acc. = 0.76174; Elapsed time = 0.66 s
____________________
Session length = 15
Window size = 5
CV acc. = 0.86658; Val. acc. = 0.87530; Elapsed time = 2.70 s
____________________
Session length = 10
Window size = 5
CV acc. = 0.81755; Val. acc. = 0.82432; Elapsed time = 1.55 s
____________________
Session length = 7
Window size = 5
CV acc. = 0.77274; Val. acc. = 0.78509; Elapsed time = 1.08 s
____________________
Session length = 5
Window size = 5
CV acc. = 0.72543; Val. acc. = 0.73578; Elapsed time = 0.77 s
____________________


estimator = svm_cv.best_estimator_

for window_size, session_length in [(5,5), (7,7), (10,10)]:
    X_file_name = f'X_sparse_150users_s{session_length}_w{window_size}.pkl'
    y_file_name = f'y_150users_s{session_length}_w{window_size}.pkl'
    path_to_X_pkl = os.path.join(PATH_TO_DATA, X_file_name)
    path_to_y_pkl = os.path.join(PATH_TO_DATA, y_file_name)
    print(f'Session length = {session_length}')
    print(f'Window size = {window_size}')
    print('CV acc. = {0:.5f}; Val. acc. = {1:.5f}; Elapsed time = {2}' \
          .format(*model_assessment(estimator, 
                                    path_to_X_pkl, path_to_y_pkl, skf,
                                    random_state=17, test_size=0.3)))
    print('_' * 20)

Session length = 5
Window size = 5
CV acc. = 0.40818; Val. acc. = 0.42161; Elapsed time = 163.71 s
____________________
Session length = 7
Window size = 7
CV acc. = 0.43633; Val. acc. = 0.45263; Elapsed time = 157.99 s
____________________
Session length = 10
Window size = 10
CV acc. = 0.46291; Val. acc. = 0.48358; Elapsed time = 156.25 s
____________________


train_df = pd.read_csv(os.path.join(PATH_TO_DATA, 'train_sessions.csv'),
                       index_col='session_id')
test_df = pd.read_csv(os.path.join(PATH_TO_DATA, 'test_sessions.csv'),
                      index_col='session_id')


train_df.head()


train_test_df = pd.concat([train_df, test_df])


train_df.info()

<class 'pandas.core.frame.DataFrame'>
Int64Index: 253561 entries, 1 to 253561
Data columns (total 21 columns):
 #   Column  Non-Null Count   Dtype  
---  ------  --------------   -----  
 0   site1   253561 non-null  int64  
 1   time1   253561 non-null  object 
 2   site2   250098 non-null  float64
 3   time2   250098 non-null  object 
 4   site3   246919 non-null  float64
 5   time3   246919 non-null  object 
 6   site4   244321 non-null  float64
 7   time4   244321 non-null  object 
 8   site5   241829 non-null  float64
 9   time5   241829 non-null  object 
 10  site6   239495 non-null  float64
 11  time6   239495 non-null  object 
 12  site7   237297 non-null  float64
 13  time7   237297 non-null  object 
 14  site8   235224 non-null  float64
 15  time8   235224 non-null  object 
 16  site9   233084 non-null  float64
 17  time9   233084 non-null  object 
 18  site10  231052 non-null  float64
 19  time10  231052 non-null  object 
 20  target  253561 non-null  int64  
dtypes: float64(9), int64(2), object(10)
memory usage: 42.6+ MB


test_df.head()


test_df.info()

<class 'pandas.core.frame.DataFrame'>
Int64Index: 82797 entries, 1 to 82797
Data columns (total 20 columns):
 #   Column  Non-Null Count  Dtype  
---  ------  --------------  -----  
 0   site1   82797 non-null  int64  
 1   time1   82797 non-null  object 
 2   site2   81308 non-null  float64
 3   time2   81308 non-null  object 
 4   site3   80075 non-null  float64
 5   time3   80075 non-null  object 
 6   site4   79182 non-null  float64
 7   time4   79182 non-null  object 
 8   site5   78341 non-null  float64
 9   time5   78341 non-null  object 
 10  site6   77566 non-null  float64
 11  time6   77566 non-null  object 
 12  site7   76840 non-null  float64
 13  time7   76840 non-null  object 
 14  site8   76151 non-null  float64
 15  time8   76151 non-null  object 
 16  site9   75484 non-null  float64
 17  time9   75484 non-null  object 
 18  site10  74806 non-null  float64
 19  time10  74806 non-null  object 
dtypes: float64(9), int64(1), object(10)
memory usage: 13.3+ MB


train_df['target'].value_counts()

0    251264
1      2297
Name: target, dtype: int64


train_test_df_sites = train_test_df[['site%d' % i for i in range(1, 11)]].fillna(0).astype('int')


train_test_df_sites.head(10)


%%time
try:
    with open(os.path.join(PATH_TO_DATA, 'train_test_sparse.pkl'), 'rb') as train_test_sparse_pkl:
        train_test_sparse = pickle.load(train_test_sparse_pkl)
    with open(os.path.join(PATH_TO_DATA, 'X_train_sparse.pkl'), 'rb') as X_train_sparse_pkl:
        X_train_sparse = pickle.load(X_train_sparse_pkl)
    with open(os.path.join(PATH_TO_DATA, 'X_test_sparse.pkl'), 'rb') as X_test_sparse_pkl:
        X_test_sparse = pickle.load(X_test_sparse_pkl)
    with open(os.path.join(PATH_TO_DATA, 'train_target.pkl'), 'rb') as train_target_pkl:
        y = pickle.load(train_target_pkl)
except FileNotFoundError:
    print('Files not found. Creating.')

    train_test_sparse = get_sparse_matrix(train_test_df_sites.values)
    X_train_sparse = train_test_sparse[:len(train_df)]
    X_test_sparse = train_test_sparse[-len(test_df):]
    y = train_df.target.values

    with open(os.path.join(PATH_TO_DATA, 'train_test_sparse.pkl'), 'wb') as train_test_sparse_pkl:
        pickle.dump(train_test_sparse, train_test_sparse_pkl)
    with open(os.path.join(PATH_TO_DATA, 'X_train_sparse.pkl'), 'wb') as X_train_sparse_pkl:
        pickle.dump(X_train_sparse, X_train_sparse_pkl)
    with open(os.path.join(PATH_TO_DATA, 'X_test_sparse.pkl'), 'wb') as X_test_sparse_pkl:
        pickle.dump(X_test_sparse, X_test_sparse_pkl)
    with open(os.path.join(PATH_TO_DATA, 'train_target.pkl'), 'wb') as train_target_pkl:
        pickle.dump(y, train_target_pkl)

CPU times: user 14.9 ms, sys: 20 ms, total: 34.9 ms
Wall time: 697 ms


print('Размерности матриц')
print(f'Train: {X_train_sparse.shape}; Test: {X_test_sparse.shape}')

Размерности матриц
Train: (253561, 48371); Test: (82797, 48371)


train_share = int(.7 * X_train_sparse.shape[0])
X_train, y_train = X_train_sparse[:train_share, :], y[:train_share]
X_valid, y_valid  = X_train_sparse[train_share:, :], y[train_share:]


sgd_logit = SGDClassifier(loss='log', random_state=17, n_jobs=-1)
sgd_logit.fit(X_train, y_train);


logit_valid_pred_proba = sgd_logit.predict_proba(X_valid)[:, 1]
sgd_logit_roc_auc = roc_auc_score(y_valid, logit_valid_pred_proba)
print(f'SGD Logit ROC AUC = {sgd_logit_roc_auc:.5f}')

SGD Logit ROC AUC = 0.93362


%%time
sgd_logit.fit(X_train_sparse, y)
logit_test_pred_proba = sgd_logit.predict_proba(X_test_sparse)[:, 1]

CPU times: user 1.08 s, sys: 23.6 ms, total: 1.11 s
Wall time: 482 ms


def write_to_submission_file(predicted_labels, out_file,
                             target='target', index_label="session_id"):
    # turn predictions into data frame and save as csv file
    predicted_df = pd.DataFrame(predicted_labels,
                                index = np.arange(1, predicted_labels.shape[0] + 1),
                                columns=[target])
    predicted_df.to_csv(out_file, index_label=index_label)


write_to_submission_file(logit_test_pred_proba, 'sgdbase.csv')


def new_features1(data):
    
    feature_names = ['time_diff%i' % i for i in range(1, 10)] + \
                    ['session_timespan', 'unique_sites', 'start_hour', 'day_of_week']
    diffs = []
    timespans = []
    unique_sites = []
    start_hours = []
    days_of_week = []
    times_columns = ['time%i' % i for i in range(1, 11)]
    sites_columns = ['site%i' % i for i in range(1, 11)]
    
    # Заполняем нулями все пропущенные сайты, а все пропущенные времена заполняем последним известным
    data.loc[:, times_columns] = data.loc[:, times_columns].ffill(axis=1).astype('datetime64')
    data.loc[:, sites_columns] = data.loc[:, sites_columns].fillna(0).astype('int')
    
    for ind in tqdm(range(len(data))):
        row = data.iloc[ind]
        # Получаем количество уникальных сайтов в сессии
        num_unique = len({x for x in row.loc[sites_columns] if x})
        # Получаем времена захода на сайты
        timestamps = row.loc[times_columns]
        # Получаем разницу времени между заходом на следующий сайт (информация по последнему сайту неизвестна)
        diff = list(map(lambda x: x.total_seconds(), np.diff(timestamps)))
        
        # Получаем время сессии. Будем её считать как разницу между заходом на первый и на последний сайт
        # Мы знаем, что сессия максимально длится полчаса, поэтому если у сайта 10 id 0, то прошли 30 минут
        # Если сайт не 0, то просто считаем, что пользователь там провёл 0 секунд, хотя он вполне мог бы там пробыть до конца 30 минут
        # Но попробуем такое допущение, иначе смысла в признаке не будет
        timespan = (row['time10'] - row['time1']).total_seconds() if row['site10'] else 30*60
        hour = row['time1'].hour
        dow = row['time1'].dayofweek
        
        # Создаём строки и столбцы будущего DataFrame
        diffs.append(diff)
        timespans.append(timespan)
        unique_sites.append(num_unique)
        start_hours.append(hour)
        days_of_week.append(dow)

    # Создаём DataFrame из разниц времени между переходами на следующий сайт
    df = pd.DataFrame(diffs)

    # Добавляем остальные созданные столбцы и даём им имена
    other_features_df = pd.DataFrame(np.stack([timespans, unique_sites,
                                                start_hours, days_of_week], axis=1)) 

    df = pd.concat([df, other_features_df], axis=1).astype('int')
    df.columns = feature_names

    return df


# Загрузка новых признаков из файла либо их создание+запись
try:
    with open(os.path.join(PATH_TO_DATA, 'train_test_new_feat1.pkl'), 'rb') as new_feat1_file:
        new_feat_df1 = pickle.load(new_feat1_file)
except FileNotFoundError:
    print('File not found. Creating.')
    new_feat_df1 = new_features1(train_test_df)
    with open(os.path.join(PATH_TO_DATA, 'train_test_new_feat1.pkl'), 'wb') as new_feat1_file:
        pickle.dump(new_feat_df1, new_feat1_file)


new_feat_df1.head()


new_feat1_train = new_feat_df1[:len(train_df)]
new_feat1_test = new_feat_df1[-len(test_df):]
sns.set(font_scale=1.5, rc={'figure.figsize': (12,7)})


time_diffs = ['time_diff%i' % i for i in range(1, 10)]
mean_time_diffs_train = new_feat1_train[time_diffs].mean(axis=1)
mean_time_diffs_test = new_feat1_test[time_diffs].mean(axis=1)
print(f'Train:\n{mean_time_diffs_train.describe()}\n\nTest:\n{mean_time_diffs_test.describe()}')

Train:
count    253561.000000
mean         15.388263
std          32.864638
min           0.000000
25%           0.777778
50%           3.000000
75%          12.555556
max         200.000000
dtype: float64

Test:
count    82797.000000
mean        14.894466
std         31.684195
min          0.000000
25%          0.777778
50%          3.444444
75%         12.222222
max        200.000000
dtype: float64


sns.histplot(mean_time_diffs_train, bins=100, stat='density', color='red', label='Train', alpha=0.5)
sns.histplot(mean_time_diffs_test, bins=100, stat='density', color='green', label='Test', alpha=0.5)
plt.xlim(0, 50)
plt.xlabel('Среднее время на сайте, с.')
plt.ylabel('Доля сессий')
plt.title('Распределение среднего времени на сайте в сессии')
plt.legend();


mean_time_diffs_train_wt = pd.concat([mean_time_diffs_train, pd.Series(y)], axis=1)
mtdt_not_alice = mean_time_diffs_train_wt[mean_time_diffs_train_wt[1] == 0][0]
mtdt_alice = mean_time_diffs_train_wt[mean_time_diffs_train_wt[1] == 1][0]


sns.histplot(mtdt_not_alice, color='red', stat='density', bins=100, label='Not Alice', alpha=0.5)
sns.histplot(mtdt_alice, color='green', stat='density', bins=100, label='Alice', alpha=0.5)
plt.xlim(0, 50)
plt.xlabel('Среднее время на сайте, с.')
plt.ylabel('Доля сессий')
plt.title('Распределение среднего времени на сайте в сессии')
plt.legend();


sess_time_train = new_feat1_train['session_timespan']
sess_time_test = new_feat1_test['session_timespan']
print(f'Train:\n{sess_time_train.describe()}\n\nTest:\n{sess_time_test.describe()}')

Train:
count    253561.000000
mean        268.747777
std         535.319084
min           0.000000
25%           8.000000
50%          34.000000
75%         163.000000
max        1800.000000
Name: session_timespan, dtype: float64

Test:
count    82797.000000
mean       278.269901
std        547.561456
min          0.000000
25%         10.000000
50%         40.000000
75%        163.000000
max       1800.000000
Name: session_timespan, dtype: float64


sns.histplot(sess_time_train, stat='density', bins=60, color='red', label='Train', alpha=0.5)
sns.histplot(sess_time_test, stat='density', bins=60, color='green', label='Test', alpha=0.5)
plt.xlabel('Время сессии, с.')
plt.ylabel('Доля сессий')
plt.title('Распределение времени сессии')
plt.legend();


sess_time_train_wt = pd.concat([sess_time_train, pd.Series(y)], axis=1)
st_not_alice = sess_time_train_wt[sess_time_train_wt[0] == 0]['session_timespan']
st_alice = sess_time_train_wt[sess_time_train_wt[0] == 1]['session_timespan']


sns.histplot(st_not_alice, color='red', stat='density', bins=60, label='Not Alice', alpha=0.5)
sns.histplot(st_alice, color='green', stat='density', bins=60, label='Alice', alpha=0.5)
plt.xlabel('Время сессии, с.')
plt.ylabel('Доля сессий')
plt.title('Распределение времени сессии')
plt.legend();


unique_sites_train = new_feat1_train['unique_sites']
unique_sites_test = new_feat1_test['unique_sites']
print(f'Train:\n{unique_sites_train.describe()}\n\nTest:\n{unique_sites_test.describe()}')

Train:
count    253561.000000
mean          5.638391
std           2.496187
min           1.000000
25%           4.000000
50%           6.000000
75%           8.000000
max          10.000000
Name: unique_sites, dtype: float64

Test:
count    82797.000000
mean         5.280650
std          2.626599
min          1.000000
25%          3.000000
50%          5.000000
75%          7.000000
max         10.000000
Name: unique_sites, dtype: float64


sns.histplot(unique_sites_train, stat='density',  color='red', label='Train', alpha=0.5, discrete=True)
sns.histplot(unique_sites_test, stat='density',  color='green', label='Test', alpha=0.5, discrete=True)
plt.xlabel('Число уникальных сайтов')
plt.ylabel('Доля сессий')
plt.title('Распределение числа уникальных сайтов')
plt.xticks(range(unique_sites_train.min(), unique_sites_train.max()+1))
plt.legend();


unique_sites_train_wt = pd.concat([unique_sites_train, pd.Series(y)], axis=1)
us_not_alice = unique_sites_train_wt[unique_sites_train_wt[0] == 0]['unique_sites']
us_alice = unique_sites_train_wt[unique_sites_train_wt[0] == 1]['unique_sites']


sns.histplot(us_not_alice, color='red', stat='density', label='Not Alice', alpha=0.5, discrete=True)
sns.histplot(us_alice, color='green', stat='density', label='Alice', alpha=0.5, discrete=True)
plt.xlabel('Число уникальных сайтов')
plt.ylabel('Доля сессий')
plt.title('Распределение числа уникальных сайтов')
plt.xticks(range(unique_sites_train.min(), unique_sites_train.max()+1))
plt.legend();


start_hour_train = new_feat1_train['start_hour']
start_hour_test = new_feat1_test['start_hour']
print(f'Train:\n{start_hour_train.describe()}\n\nTest:\n{start_hour_test.describe()}')

Train:
count    253561.000000
mean         12.288483
std           3.159420
min           7.000000
25%          10.000000
50%          12.000000
75%          15.000000
max          23.000000
Name: start_hour, dtype: float64

Test:
count    82797.000000
mean        12.504064
std          3.082344
min          7.000000
25%         10.000000
50%         12.000000
75%         15.000000
max         23.000000
Name: start_hour, dtype: float64


sns.histplot(start_hour_train, stat='density',  color='red', label='Train', alpha=0.5, discrete=True)
sns.histplot(start_hour_test, stat='density',  color='green', label='Test', alpha=0.5, discrete=True)
plt.xlabel('Час начала')
plt.ylabel('Доля сессий')
plt.title('Распределение часа начала сессии')
plt.xticks(range(start_hour_train.min(), start_hour_train.max()+1))
plt.legend();


start_hour_train_wt = pd.concat([start_hour_train, pd.Series(y)], axis=1)
sh_not_alice = start_hour_train_wt[start_hour_train_wt[0] == 0]['start_hour']
sh_alice = start_hour_train_wt[start_hour_train_wt[0] == 1]['start_hour']


sns.histplot(sh_not_alice, color='red', stat='density', label='Not Alice', alpha=0.5, discrete=True)
sns.histplot(sh_alice, color='green', stat='density', label='Alice', alpha=0.5, discrete=True)
plt.xlabel('Час начала')
plt.ylabel('Доля сессий')
plt.title('Распределение часа начала сессии')
plt.xticks(range(start_hour_train.min(), start_hour_train.max()+1))
plt.legend();


day_of_week_train = new_feat1_train['day_of_week']
day_of_week_test = new_feat1_test['day_of_week']
print(f'Train:\n{day_of_week_train.describe()}\n\nTest:\n{day_of_week_test.describe()}')

Train:
count    253561.000000
mean          2.289741
std           1.610467
min           0.000000
25%           1.000000
50%           2.000000
75%           4.000000
max           6.000000
Name: day_of_week, dtype: float64

Test:
count    82797.000000
mean         2.963598
std          1.941522
min          0.000000
25%          1.000000
50%          3.000000
75%          4.000000
max          6.000000
Name: day_of_week, dtype: float64


sns.histplot(day_of_week_train, stat='density',  color='red', label='Train', alpha=0.5, discrete=True)
sns.histplot(day_of_week_test, stat='density',  color='green', label='Test', alpha=0.5, discrete=True)
plt.xlabel('День недели')
plt.ylabel('Доля сессий')
plt.title('Распределение дня недели')
plt.xticks(range(7), ['Пн', 'Вт', 'Ср', 'Чт', 'Пт', 'Сб', 'Вс'])
plt.legend();


day_of_week_train_wt = pd.concat([day_of_week_train, pd.Series(y)], axis=1)
dow_not_alice = day_of_week_train_wt[day_of_week_train_wt[0] == 0]['day_of_week']
dow_alice = day_of_week_train_wt[day_of_week_train_wt[0] == 1]['day_of_week']


sns.histplot(dow_not_alice, stat='density',  color='red', label='Not Alice', alpha=0.5, discrete=True)
sns.histplot(dow_alice, stat='density',  color='green', label='Alice', alpha=0.5, discrete=True)
plt.xlabel('День недели')
plt.ylabel('Доля сессий')
plt.title('Распределение дня недели')
plt.xticks(range(7), ['Пн', 'Вт', 'Ср', 'Чт', 'Пт', 'Сб', 'Вс'])
plt.legend();


# Добавим признаки в разреженную матрицу и поделим на train-test
def add_features_to_sparse(full_data, new_features, train_size):
    '''
    Add features to sparse matrices and split into train and test
    '''
    if new_features.ndim == 1:
        new_features = new_features.reshape(-1, 1)
    X_sparse_full = hstack([full_data, new_features], format='csr')
    X_train = X_sparse_full[:train_size]
    X_test = X_sparse_full[train_size:]
    return X_train, X_test


def plot_learning_curve(estimator, X_train, y_train, cv=3, scoring='roc_auc'):
    '''
    Plot learning curves for estimator
    '''
    train_sizes = [0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 1]
    sizes, train_scores, test_scores = learning_curve(estimator, X_train, y_train, train_sizes=train_sizes,
                                                      cv=cv, scoring=scoring, random_state=17)
    sns.lineplot(x=sizes, y=train_scores.mean(axis=1), color='red', label='Train')
    sns.lineplot(x=sizes, y=test_scores.mean(axis=1), color='green', label='Test')
    plt.xlabel('Train size')
    plt.ylabel('ROC-AUC')
    plt.legend();


def get_val_score(estimator, X, y, X_val, y_val):
    '''
    Fit estimator and get validation set score
    '''
    estimator.fit(X_train, y_train)
    val_pred = estimator.predict_proba(X_val)[:, 1]
    print(f'Validation ROC-AUC score = {roc_auc_score(y_val, val_pred):.5f}')


sgd_new1 = SGDClassifier(loss='log', random_state=17, n_jobs=-1)


add_feat = ['start_hour', 'day_of_week']
X_train_new, X_test_new = add_features_to_sparse(train_test_sparse, new_feat_df1[add_feat].values, len(train_df))
X_train, X_val, y_train, y_val = train_test_split(X_train_new, y, test_size=0.3, random_state=17, shuffle=False)    
get_val_score(sgd_new1, X_train, y_train, X_val, y_val)
plot_learning_curve(sgd_new1, X_train, y_train)

Validation ROC-AUC score = 0.88585


time_of_day = pd.get_dummies(build_time_of_day(new_feat_df1.start_hour))
is_weekend = build_is_weekend(new_feat_df1.day_of_week)
new_feat2 = pd.concat([time_of_day, is_weekend], axis=1)
new_feat2


X_train_new, X_test_new = add_features_to_sparse(train_test_sparse, new_feat2.values, len(train_df))
X_train, X_val, y_train, y_val = train_test_split(X_train_new, y, test_size=0.3, random_state=17, shuffle=False)    
get_val_score(sgd_new1, X_train, y_train, X_val, y_val)
plot_learning_curve(sgd_new1, X_train, y_train)

Validation ROC-AUC score = 0.96252


%%time
sgd_new1.fit(X_train_new, y)
test_pred = sgd_new1.predict_proba(X_test_new)[:, 1]
write_to_submission_file(test_pred, 'sgd2feat1hot.csv')

CPU times: user 1.49 s, sys: 95.3 ms, total: 1.59 s
Wall time: 699 ms


X_train_new, X_test_new = add_features_to_sparse(train_test_sparse, new_feat_df1.values, len(train_df))
X_train, X_val, y_train, y_val = train_test_split(X_train_new, y, test_size=0.3, random_state=17, shuffle=False)    
get_val_score(sgd_new1, X_train, y_train, X_val, y_val)
plot_learning_curve(sgd_new1, X_train, y_train)

Validation ROC-AUC score = 0.73873


def scale_features(data, train_size):
    scaler = MinMaxScaler()
    data_train = data.iloc[:train_size, :]
    data_test = data.iloc[train_size:, :]

    X_train, X_val = train_test_split(data_train, test_size=0.3, random_state=17, shuffle=False)

    X_train_scaled = scaler.fit_transform(X_train)
    X_val_scaled = scaler.transform(X_val)
    X_test_scaled = scaler.transform(data_test)

    return np.vstack([X_train_scaled, X_val_scaled, X_test_scaled])


feat_to_scale = ['session_timespan', 'unique_sites']
to_scale_df = new_feat_df1[feat_to_scale]
scaled_features = scale_features(to_scale_df, len(train_df))
new_feat3 = np.hstack([new_feat2.values, scaled_features])


X_train_new, X_test_new = add_features_to_sparse(train_test_sparse, new_feat3, len(train_df))
X_train, X_val, y_train, y_val = train_test_split(X_train_new, y, test_size=0.3, random_state=17, shuffle=False)    
get_val_score(sgd_new1, X_train, y_train, X_val, y_val)
plot_learning_curve(sgd_new1, X_train, y_train)

Validation ROC-AUC score = 0.96137


param_grid = {'loss': ['hinge', 'log'],
              'alpha': np.logspace(-1, -7, 14),
              'l1_ratio': np.linspace(0, 1, 5),
              'class_weight': [None, 'balanced']
              }
cv_est = SGDClassifier(loss='log', random_state=17, n_jobs=-1, penalty='elasticnet')
gs_sgd_new = GridSearchCV(cv_est, param_grid=param_grid, scoring='roc_auc',
                        verbose=1, n_jobs=-1, return_train_score=True, cv=3)


gs_sgd_new.fit(X_train, y_train)

Fitting 3 folds for each of 280 candidates, totalling 840 fits

[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.
[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:    4.2s
[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.4min
[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  5.0min
[Parallel(n_jobs=-1)]: Done 792 tasks      | elapsed: 10.1min
[Parallel(n_jobs=-1)]: Done 840 out of 840 | elapsed: 10.8min finished

GridSearchCV(cv=3,
             estimator=SGDClassifier(loss='log', n_jobs=-1,
                                     penalty='elasticnet', random_state=17),
             n_jobs=-1,
             param_grid={'alpha': array([1.00000000e-01, 3.45510729e-02, 1.19377664e-02, 4.12462638e-03,
       1.42510267e-03, 4.92388263e-04, 1.70125428e-04, 5.87801607e-05,
       2.03091762e-05, 7.01703829e-06, 2.42446202e-06, 8.37677640e-07,
       2.89426612e-07, 1.00000000e-07]),
                         'class_weight': [None, 'balanced'],
                         'l1_ratio': array([0.  , 0.25, 0.5 , 0.75, 1.  ]),
                         'loss': ['hinge', 'log']},
             return_train_score=True, scoring='roc_auc', verbose=1)


gs_sgd_new.best_params_

{'alpha': 0.00017012542798525892,
 'class_weight': 'balanced',
 'l1_ratio': 0.0,
 'loss': 'log'}


get_val_score(gs_sgd_new.best_estimator_, X_train, y_train, X_val, y_val)
plot_learning_curve(gs_sgd_new.best_estimator_, X_train, y_train)

Validation ROC-AUC score = 0.96936


gs_sgd_new.best_estimator_.fit(X_train_new, y)
test_pred = gs_sgd_new.best_estimator_.predict_proba(X_test_new)[:, 1]
write_to_submission_file(test_pred, 'allfeat_gs.csv')

	timestamp	site
0	2013-11-15 08:12:07	fpdownload2.macromedia.com
1	2013-11-15 08:12:17	laposte.net
2	2013-11-15 08:12:17	www.laposte.net
3	2013-11-15 08:12:17	www.google.com
4	2013-11-15 08:12:18	www.laposte.net

	site1	time1	site2	time2	site3	time3	site4	time4	site5	time5	site6	time6	site7	time7	site8	time8	site9	time9	site10	time10	target
session_id
1	718	2014-02-20 10:02:45	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	0
2	890	2014-02-22 11:19:50	941.0	2014-02-22 11:19:50	3847.0	2014-02-22 11:19:51	941.0	2014-02-22 11:19:51	942.0	2014-02-22 11:19:51	3846.0	2014-02-22 11:19:51	3847.0	2014-02-22 11:19:52	3846.0	2014-02-22 11:19:52	1516.0	2014-02-22 11:20:15	1518.0	2014-02-22 11:20:16	0
3	14769	2013-12-16 16:40:17	39.0	2013-12-16 16:40:18	14768.0	2013-12-16 16:40:19	14769.0	2013-12-16 16:40:19	37.0	2013-12-16 16:40:19	39.0	2013-12-16 16:40:19	14768.0	2013-12-16 16:40:20	14768.0	2013-12-16 16:40:21	14768.0	2013-12-16 16:40:22	14768.0	2013-12-16 16:40:24	0
4	782	2014-03-28 10:52:12	782.0	2014-03-28 10:52:42	782.0	2014-03-28 10:53:12	782.0	2014-03-28 10:53:42	782.0	2014-03-28 10:54:12	782.0	2014-03-28 10:54:42	782.0	2014-03-28 10:55:12	782.0	2014-03-28 10:55:42	782.0	2014-03-28 10:56:12	782.0	2014-03-28 10:56:42	0
5	22	2014-02-28 10:53:05	177.0	2014-02-28 10:55:22	175.0	2014-02-28 10:55:22	178.0	2014-02-28 10:55:23	177.0	2014-02-28 10:55:23	178.0	2014-02-28 10:55:59	175.0	2014-02-28 10:55:59	177.0	2014-02-28 10:55:59	177.0	2014-02-28 10:57:06	178.0	2014-02-28 10:57:11	0

	site1	time1	site2	time2	site3	time3	site4	time4	site5	time5	site6	time6	site7	time7	site8	time8	site9	time9	site10	time10
session_id
1	29	2014-10-04 11:19:53	35.0	2014-10-04 11:19:53	22.0	2014-10-04 11:19:54	321.0	2014-10-04 11:19:54	23.0	2014-10-04 11:19:54	2211.0	2014-10-04 11:19:54	6730.0	2014-10-04 11:19:54	21.0	2014-10-04 11:19:54	44582.0	2014-10-04 11:20:00	15336.0	2014-10-04 11:20:00
2	782	2014-07-03 11:00:28	782.0	2014-07-03 11:00:53	782.0	2014-07-03 11:00:58	782.0	2014-07-03 11:01:06	782.0	2014-07-03 11:01:09	782.0	2014-07-03 11:01:10	782.0	2014-07-03 11:01:23	782.0	2014-07-03 11:01:29	782.0	2014-07-03 11:01:30	782.0	2014-07-03 11:01:53
3	55	2014-12-05 15:55:12	55.0	2014-12-05 15:55:13	55.0	2014-12-05 15:55:14	55.0	2014-12-05 15:56:15	55.0	2014-12-05 15:56:16	55.0	2014-12-05 15:56:17	55.0	2014-12-05 15:56:18	55.0	2014-12-05 15:56:19	1445.0	2014-12-05 15:56:33	1445.0	2014-12-05 15:56:36
4	1023	2014-11-04 10:03:19	1022.0	2014-11-04 10:03:19	50.0	2014-11-04 10:03:20	222.0	2014-11-04 10:03:21	202.0	2014-11-04 10:03:21	3374.0	2014-11-04 10:03:22	50.0	2014-11-04 10:03:22	48.0	2014-11-04 10:03:22	48.0	2014-11-04 10:03:23	3374.0	2014-11-04 10:03:23
5	301	2014-05-16 15:05:31	301.0	2014-05-16 15:05:32	301.0	2014-05-16 15:05:33	66.0	2014-05-16 15:05:39	67.0	2014-05-16 15:05:40	69.0	2014-05-16 15:05:40	70.0	2014-05-16 15:05:40	68.0	2014-05-16 15:05:40	71.0	2014-05-16 15:05:40	167.0	2014-05-16 15:05:44

	site1	site2	site3	site4	site5	site6	site7	site8	site9	site10
session_id
1	718	0	0	0	0	0	0	0	0	0
2	890	941	3847	941	942	3846	3847	3846	1516	1518
3	14769	39	14768	14769	37	39	14768	14768	14768	14768
4	782	782	782	782	782	782	782	782	782	782
5	22	177	175	178	177	178	175	177	177	178
6	570	21	570	21	21	0	0	0	0	0
7	803	23	5956	17513	37	21	803	17514	17514	17514
8	22	21	29	5041	14422	23	21	5041	14421	14421
9	668	940	942	941	941	942	940	23	21	22
10	3700	229	570	21	229	21	21	21	2336	2044

	afternoon	evening	morning	is_weekend
0	0	0	1	0
1	0	0	1	1
2	1	0	0	0
3	0	0	1	0
4	0	0	1	0
...	...	...	...	...
336353	0	1	0	0
336354	1	0	0	0
336355	0	0	1	0
336356	0	0	1	1
336357	0	0	1	1

Финальный проект "Идентификация интернет-пользователей".

Проект разделен на несколько частей:¶

Часть 1.¶

Подготовим обучающую выборку¶

Модифицируем выборку¶

Результаты 1 части.¶

Часть 2.¶

Исследование данных и проверка гипотез¶

Построение признаков и визуализация данных¶

Дальнейшее построение признаков.¶

Результаты 2 части.¶

Часть 3.¶

Результаты 3 части.¶

Часть 4.¶

Пробуем улучшить результат¶

Посмотрим на визуализацию новых признаков.¶

Попробуем добавить эти признаки и обучить классификатор¶

Результаты 4 части.¶

Заключение¶

	site1	site2	site3	site4	site5	site6	site7	site8	site9	site10	user_id
0	3	2	2	7	2	1	8	5	9	10	1
1	3	1	1	1	0	0	0	0	0	0	1
2	3	2	6	6	2	0	0	0	0	0	2
3	4	1	2	1	2	1	1	5	11	4	3
4	4	1	2	0	0	0	0	0	0	0	3

	site1	site2	site3	site4	site5	site6	site7	site8	site9	site10	user_id
0	192	574	133	3	133	133	3	133	203	133	31
1	415	193	674	254	133	31	393	3305	217	55	31
2	55	3	55	55	5	293	415	333	897	55	31
3	473	3306	473	55	55	55	55	937	199	123	31
4	342	55	5	3307	258	211	3308	2086	675	2086	31

	site1	site2	site3	site4	site5	site6	site7	site8	site9	site10	time_diff1	time_diff2	time_diff3	time_diff4	time_diff5	time_diff6	time_diff7	time_diff8	time_diff9	session_timespan	#unique_sites	start_hour	day_of_week	target
0	1	1	4	2	1	4	1	2	1290	321	1	1	1	4	0	1	0	49	0	57	5	8	4	6
1	2	23	1505	1290	321	321	113	73	49	49	0	0	26	1	10	73	0	1504	4	1618	8	8	4	6
2	321	205	1	1	1	1	73	321	2	1	1	10	1	5	16	56	1917	0	4	2010	5	8	4	6
3	1	1	6998	6998	5491	6998	6998	1	35	1	1	0	1	0	118	420	1460	6	0	2006	4	9	4	6
4	1	1	76	1	58	50	58	50	83	76	4	3	0	1	0	1	0	1	0	10	5	9	4	6

	time_of_day	pop_total_ratio
0	morning	1.000000
1	morning	0.000000
2	morning	0.040796
3	morning	0.003490
4	morning	0.800000

	time_diff1	time_diff2	time_diff3	time_diff4	time_diff5	time_diff6	time_diff7	time_diff8	time_diff9	session_timespan	unique_sites	start_hour	day_of_week
0	0	0	0	0	0	0	0	0	0	1800	1	10	3
1	0	1	0	0	0	1	0	23	1	26	7	11	5
2	1	1	0	0	0	1	1	1	2	7	4	16	0
3	30	30	30	30	30	30	30	30	30	270	1	10	4
4	137	0	1	0	36	0	0	67	5	246	4	10	4

	site1	site2	site3	site4	site5	site6	site7	site8	site9	site10	user_id
0	3	2	2	7	2	1	8	5	9	10	1
1	3	1	1	1	0	0	0	0	0	0	1
2	3	2	6	6	2	0	0	0	0	0	2
3	4	1	2	1	2	1	1	5	11	4	3
4	4	1	2	0	0	0	0	0	0	0	3

	site1	site2	site3	site4	site5	site6	site7	site8	site9	site10	user_id
0	3	2	2	7	2	1	8	5	9	10	1
1	3	1	1	1	0	0	0	0	0	0	1
2	3	2	6	6	2	0	0	0	0	0	2
3	4	1	2	1	2	1	1	5	11	4	3
4	4	1	2	0	0	0	0	0	0	0	3